from datetime import time

import feapder
from feapder import Item
from feapder.db.mongodb import MongoDB
from lxml import etree

from items import dangdang_item

db = MongoDB(
    ip="server.windows", port=27017, db="feapder", user_name="root", user_pass="141535"
)


class Category(object):
    # 使用装饰器模式，会自动添加一个type_one属性。当调用该属性时，自动调用该装饰的方法
    def __init__(self, type_one: str, type_two: str, url: str, datetime: str, details_list: [str]):
        self.__url = url
        self.__type_one = type_one
        self.__type_two = type_two
        self.__datetime = datetime
        self.__details_list = details_list

    def __str__(self):
        return "url:" + self.__url + ", type_one:" + self.__type_one + ", type_two: " + self.__type_two

    @property
    def url(self):
        return self.__url

    @url.setter
    def url(self, value):
        if not isinstance(value, str):
            raise ValueError('url must be a str!')
        self.__url = value

    @property
    def type_one(self):
        return self.__type_one

    @type_one.setter
    def type_one(self, value):
        if not isinstance(value, str):
            raise ValueError('type_one must be a str!')
        self.__type_one = value

    @property
    def type_two(self):
        return self.__type_two

    @type_two.setter
    def type_two(self, value):
        if not isinstance(value, str):
            raise ValueError('type_two must be a str!')
        self.__type_two = value


class Book(object):
    """
        书籍对象
        """

    def __init__(self, kind: str, url: str,
                 name: str, main_title: str, subheading: str,
                 authors: str, publisher: str, publish_date: str, number_comment: int, price: float,
                 isbn: str,
                 recommended_reason: str, description: str, media_comment: str):
        self.kind = kind
        # 详情地址
        self.url = url
        # 书名
        self.name = name
        # 标题
        self.main_title = main_title
        self.subheading = subheading
        # 作者
        self.authors = authors
        # 出版社
        self.publisher = publisher
        # 出版社时间
        self.publish_date = publish_date
        # 评论数
        self.number_comment = number_comment
        # 价格
        self.price = price
        # 国际标准书号
        self.isbn = isbn
        # 编辑推荐理由
        self.recommended_reason = recommended_reason
        # 内容简介
        self.description = description
        # 媒体评论
        self.media_comment = media_comment

        @property
        def recommended_reason(self):
            return self.recommended_reason

        @recommended_reason.setter
        def recommended_reason(self, value):
            if not isinstance(value, str):
                raise ValueError('recommended_reason must be a str!')
            self.recommended_reason = value

        @property
        def description(self):
            return self.description

        @description.setter
        def description(self, value):
            if not isinstance(value, str):
                raise ValueError('description must be a str!')
            self.description = value

        @property
        def media_comment(self):
            return self.media_comment

        @media_comment.setter
        def media_comment(self, value):
            if not isinstance(value, str):
                raise ValueError('media_comment must be a str!')
            self.media_comment = value

    def __str__(self):
        return "kind:" + self.kind + ", url:" + self.url + ", name: " + self.name \
               + ", main_title:" + self.main_title + ", subheading: " + self.subheading \
               + ", authors: " + self.authors + ", publisher:" + self.publisher + ", publish_date: " + self.publish_date + ", number_comment:" + self.number_comment \
               + ", price: " + self.price + ", isbn: " + self.isbn + ", recommended_reason:" + self.recommended_reason + ", description: " + self.description + ", media_comment:" + self.media_comment


class DangDangAirSpider(object):
    # __custom_setting__ = dict(
    #     # ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"],
    #     MONGO_IP="server.windows",
    #     MONGO_PORT=27017,
    #     MONGO_DB="feapder",
    #     MONGO_USER_NAME="root",
    #     MONGO_USER_PASS="141535",
    # )

    # 初始任务下发入口
    def start_requests(self):
        # 请求分类菜单
        yield feapder.Request("https://category.dangdang.com/")

    # 解析结果
    def parse(self, request, response):
        result = []
        datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # 所有分类的父div
        div_base = response.xpath('//div[@id="floor_1"][1]')
        # 分类div集合
        list_div_category = div_base.xpath('./div[@class="classify_kind"]')

        for div_category in list_div_category:
            # typeone内容
            type_one = div_category.xpath('./div[@class="classify_kind_name"][1]/a/text()').extract_first()
            # typetwo对象
            li_list = div_category.xpath('./ul[1]/li')
            for li in li_list:
                url = li.xpath('./a[1]/@href').extract_first()
                type_two = li.xpath('./a[1]/text()').extract_first()
                if '更多' != type_two:
                    category = Category(type_one, type_two, url, datetime, [])
                    result.append(category)
                    item = dangdang_item.DangDangCategoryItem()
                    item.table_name="dangdang_category"
                    yield item

        for r in result:
            for i in range(1, 2):
                if i != 1:
                    yield feapder.Request(r.url.replace("/cp", "/pg{}-cp".format(i)),
                                          callback=self.parse_book_list, category=r)
                else:
                    yield feapder.Request(r.url, callback=self.parse_book_list, category=r)

    def parse_book_list(self, request, response):
        # 取url
        category = request.category
        # 解析正文
        li_list = response.xpath('//ul[@class="bigimg"][1]/li')
        for li in li_list:
            details_url = li.xpath('./a[1]/@href').extract_first()

            yield feapder.Request(details_url, callback=self.parse_book,
                                  category=category)

    def xxx(self, request):

        """
        我是自定义的下载中间件
        :param request:
        :return:
        """
        request.headers = {
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "Connection": "keep-alive",
            "Cookie": "secret_key=a9c3d30d6003fcd5327b8bffe775d819; __permanent_id=20211011104011980133198650055284598; permanent_key=20211011104011610885122984d84c50; dangdang.com=email=MTMwMTEwMjg5ODM0OTUyMUBkZG1vYmlscGhvbmVfX3VzZXIuY29t&nickname=&display_id=2626170339260&customerid=uOgMhy1W6Nm3wv8uEy8j7w==&viptype=gp2ucVP/BGU=&show_name=130%2A%2A%2A%2A8983; ddscreen=2; __visit_id=20211012094026056405484270785494093; __out_refer=1634002826%7C!%7Ccn.bing.com%7C!%7C; dest_area=country_id%3D9000%26province_id%3D111%26city_id%3D0%26district_id%3D0%26town_id%3D0; pos_9_end=1634002865286; alipay_request_from=https://login.dangdang.com/signin.aspx?returnurl=http%253A%252F%252Fproduct.dangdang.com%252F25121739.html; __rpm=s_605253.451680112839%2C451680112840.1.1634002866351%7Clogin_page.login_nolocal_mobile_div..1634002922813; USERNUM=rj1vnsSi4gNUew1LGOfFCg==; login.dangdang.com=.AYH=2021101209410710073191856&.ASPXAUTH=2wxqXq76n2tspr4828biaCflU8y2Au1AF8c0fWQUVB2fx4y92pQwKA==; ddoy=email=1301102898349521%40ddmobilphone__user.com&nickname=&agree_date=1&validatedflag=0&uname=13011028983&utype=1&.ALFG=on&.ALTM=1634002923; sessionID=pc_7a612ae629ad9c491d19bf9ddc3a5c3d89e6d3844f5f3f44dbbefb7b28b7c94e; __dd_token_id=2021101209420379540719045375b11e; LOGIN_TIME=1634002924063; pos_6_start=1634002924151; pos_6_end=1634002924698; __trace_id=20211012094225187111205257475970948",
            "Host": "product.dangdang.com",
            "Referer": request.url,
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.47",
            "X-Requested-With": "XMLHttpRequest"
        }
        return request

    def parse_book(self, request, response):
        name_info = response.xpath('//div[@class="name_info"][1]')
        messbox_info = response.xpath('//div[@class="messbox_info"][1]')

        # 取url
        category = request.category

        kind = category.type_two
        # 详情地址
        url = request.url
        # 书名
        name = "name"
        # 标题
        main_title = ""
        if len(response.xpath('//div[@id="product_info"]/div[1]/h1')) != 0:
            main_title = response.xpath('//div[@id="product_info"]/div[1]/h1/text()').extract_first().strip()

        # print("main_title=%s" % main_title)
        if len(response.xpath('//div[@id="product_info"]/div[1]/h2/span')) != 0:
            subheading = response.xpath('//div[@id="product_info"]/div[1]/h2/span[1]/text()').extract_first().strip()
        # print("subheading=%s" % subheading)
        # 作者
        authors = ''
        if len(response.xpath('//*[@id="author"]/a/text()')) != 0:
            authors = response.xpath('//*[@id="author"]/a/text()').extract_first().strip()
        # print("authors=%s" % authors)
        # 出版社
        publisher = ""
        if len(response.xpath('//*[@id="product_info"]/div[2]/span[2]/a')) != 0:
            publisher = response.xpath('//*[@id="product_info"]/div[2]/span[2]/a/text()').extract_first().strip()
        # print("publisher=%s" % publisher)
        # 出版社时间
        publish_date = ""
        if len(response.xpath('//*[@id="product_info"]/div[2]/span[3]')) != 0:
            publish_date = \
                messbox_info.xpath('//*[@id="product_info"]/div[2]/span[3]/text()').extract_first().split(":")[
                    1].strip()
        # print("publish_date=%s" % publish_date)
        # 评论数
        number_comment = 0
        if len(response.xpath('//*[@id="comm_num_down"]')) != 0:
            number_comment = response.xpath('//*[@id="comm_num_down"]/text()').extract_first().strip()

        # print("number_comment=%s" % number_comment)
        # 价格
        price = 0.00
        list_element_price = response.xpath('//p[@id="dd-price"]/text()').extract()
        if len(list_element_price) > 0:
            price = list_element_price[1].strip()
        print("price=%s" % price)
        # 国际标准书号
        isbn = response.xpath('//div[@id="detail_describe"][1]/ul[1]/li[5]/text()').extract_first().split("：")[
            1].strip()
        print("isbn=%s" % isbn)

        # 编辑推荐理由
        recommended_reason = ""
        # 内容简介
        description = ""
        # 媒体评论
        media_comment = ""
        book = Book(kind, url, name, main_title, subheading, authors, publisher, publish_date, number_comment, price,
                    isbn,
                    recommended_reason, description, media_comment)

        productId = response.re(r'productId":"(.*?)"')
        template = response.re(r'template":"(.*?)"')
        describeMap = response.re(r'describeMap":"(.*?)"')
        shopId = response.re(r'shopId":"(.*?)"')
        categoryPath = response.re(r'categoryPath":"(.*?)"')

        productId_str = "".join(productId)
        template_str = "".join(template)
        describeMap_str = "".join(describeMap)
        shopId_str = "".join(shopId)
        categoryPath_str = "".join(categoryPath)

        url = "https://product.dangdang.com/index.php?r=callback%2Fdetail&productId=" + productId_str + "&templateType=" + template_str + "&describeMap=" + describeMap_str + "&shopId=" + shopId_str + "&categoryPath=" + categoryPath_str
        yield feapder.Request(url, callback=self.parse_book2, book=book)

    def parse_book2(self, request, response):
        book = request.book
        json = response.json

        data = json.get("data")
        print("url=%s " % request.url)
        html_str = data['html']

        recommended_reason = ""
        description = ""
        media_comment = ""

        if len(html_str) != 0:
            parse_html = etree.HTML(html_str)
            list_recommended_reason = parse_html.xpath('//div[@id="abstract"][1]/div[@class="descrip"][1]/p[1]/text()')
            for p in list_recommended_reason:
                recommended_reason += p

            list_description = parse_html.xpath('//div[@id="content"][1]/div[@class="descrip"][1]/p[1]/text()')
            for p in list_description:
                description += p

            list_media_comment = parse_html.xpath('//span[@id="mediaFeedback-show"][1]/text()')
            for p in list_media_comment:
                media_comment += p

        book.recommended_reason = recommended_reason
        book.description = description
        book.media_comment = media_comment

        print(book)
        item = Item()  # 声明一个item
        item.table_name = "dangdangbook"  # 指定存储的表名
        item.kind = book.kind
        item.url = book.url
        item.name = book.name
        item.main_title = book.main_title
        item.subheading = book.subheading
        item.authors = book.authors
        item.publisher = book.publisher
        item.publish_date = book.publish_date
        item.number_comment = book.number_comment
        item.price = book.price
        item.isbn = book.isbn
        item.recommended_reason = book.recommended_reason
        item.description = book.description
        item.media_comment = book.media_comment
        item.create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        yield item  # 返回item， item会自动批量入库
